# -*- coding:utf-8 -*-
__author__ = "MiracleWong"

import requests

url = "https://mp.weixin.qq.com/mp/profile_ext?" \
      "action=home&" \
      "__biz=MzA5NTEzNzgyMA==&" \
      "scene=124&" \
      "devicetype=android-24&" \
      "version=26060339&" \
      "lang=zh_CN&nettype=WIFI&a8scene=3&" \
      "pass_ticket=lpWqtuGKXPAfQxtfVeaXziZDig53rh0jQP4I62Z37vdrPzGvdH13XlRDyiCKdqjO&" \
      "wx_header=1"
headers = """
    Host: mp.weixin.qq.com
    Cookie: sd_userid=61471515146823650; sd_cookie_crttime=1515146823650; pgv_pvi=9133319168; pgv_si=s4650095616; _scan_has_moon=1; tvfe_boss_uuid=5c58678e7ba38641; pgv_info=ssid=s3745570706&pgvReferrer=; pgv_pvid=7419517705; wxtokenkey=001a17c2b14939beb95204ed27dee251abbfc6b6888168c3c4670bd7c4dca59b; rewardsn=5607611decffbf5336f2; wxuin=794795541; devicetype=android-24; version=26060339; lang=zh_CN; pass_ticket=lpWqtuGKXPAfQxtfVeaXziZDig53rh0jQP4I62Z37vdrPzGvdH13XlRDyiCKdqjO; wap_sid2=CJW8/voCElxSSlVrYkFubmM2dEowMkRyaXZMbkxMdnBnNE80aFZwMG5pVjh5TWZmOEhpWTdZTGxFVDhTc2ZlTy1TSEFFb2FrZk10UlVWUmNRMlJMTkV2MkJLSUFLYklEQUFBfjDX/urUBTgNQJVO
    X-WECHAT-KEY: 4efbaf4659a9ceda270df73c99ce37c3db67622e10101fbba211100e76baea66f89d3d3c526612681ec6b0708fb4fbddbf8ab70cfca40a5295255a3ce8274c4ed2cad33d6b2e2f3c94f8b3b5840a5945
    X-WECHAT-UIN: Nzk0Nzk1NTQx
    Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,image/wxpic,image/sharpp,image/apng,*/*;q=0.8
    User-Agent: Mozilla/5.0 (Linux; Android 7.0; HUAWEI CAZ-AL10 Build/HUAWEICAZ-AL10; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/6.2 TBS/043906 Mobile Safari/537.36 MicroMessenger/6.6.3.1260(0x26060339) NetType/WIFI Language/zh_CN
    Accept-Language: zh-CN,zh-CN;q=0.8,en-US;q=0.6
    Accept-Encoding: gzip, deflate
    Connection: keep-alive
    Q-UA2: QV=3&PL=ADR&PR=WX&PP=com.tencent.mm&PPVN=6.6.3&TBSVC=43603&CO=BK&COVC=043906&PB=GE&VE=GA&DE=PHONE&CHID=0&LCID=9422&MO= HUAWEICAZ-AL10 &RL=1080*1788&OS=7.0&API=24
    Q-GUID: b75a82c53466e0e26d10001713b788cb
    Q-Auth: 31045b957cf33acf31e40be2f3e71c5217597676a9729f1b
"""
def crawl(headers, url):
    headers = headers_to_dict(headers)
    response = requests.get(url, headers=headers, verify=False)
    # print(response.text)
    if '<title>验证</title>' in response.text:
        raise Exception("获取微信公众号文章失败，可能是因为你的请求参数有误，请重新获取")
    data = extract_data(response.text)
    # for item in data:
    #     print(item)

def extract_data(html_content):
    """
    从html页面中提取历史文章数据
    :param html_content 页面源代码
    :return: 历史文章列表
    """
    import re
    import html
    import json

    rex = "msgList = '({.*?})'"
    pattern = re.compile(pattern=rex, flags=re.S)
    match = pattern.search(html_content)
    if match:
        data = match.group(1)
        data = html.unescape(data)
        data = json.loads(data)
        articles = data.get("list")
        # for item in articles:
        #     print(item)
        with open("caojiang.json", "a", encoding="utf-8") as f:
            for item in articles:
                print(item)
                f.write(json.dumps(item))
                f.write('\n')
        return articles

def headers_to_dict(headers):
    """
    将字符串
    '''
    Host: mp.weixin.qq.com
    Connection: keep-alive
    Cache-Control: max-age=
    '''
    转换成字典类型
    :param headers: str
    :return: dict
    """
    headers = headers.split('\n')
    d_headers = dict()
    for h in headers:
        h = h.strip()
        if h:
            k, v = h.split(":", 1)
            d_headers[k] = v.strip()
    return d_headers


def main():
    head = headers_to_dict(headers)
    for k, v in head.items():
        print('%s -- %s' % (k, v))


if __name__ == '__main__':
    crawl(headers,url)